{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2019-08-20 17:51:49.741823\n"
     ]
    }
   ],
   "source": [
    "import matplotlib\n",
    "import  matplotlib.pyplot  as plt\n",
    "import datetime \n",
    "print(datetime.datetime.now())\n",
    "%matplotlib inline\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.14.0\n"
     ]
    }
   ],
   "source": [
    "import tensorflow\n",
    "print(tensorflow.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.14.0\n",
      "((array([[[0, 0, 0, ..., 0, 0, 0],\n",
      "        [0, 0, 0, ..., 0, 0, 0],\n",
      "        [0, 0, 0, ..., 0, 0, 0],\n",
      "        ...,\n",
      "        [0, 0, 0, ..., 0, 0, 0],\n",
      "        [0, 0, 0, ..., 0, 0, 0],\n",
      "        [0, 0, 0, ..., 0, 0, 0]],\n",
      "\n",
      "       [[0, 0, 0, ..., 0, 0, 0],\n",
      "        [0, 0, 0, ..., 0, 0, 0],\n",
      "        [0, 0, 0, ..., 0, 0, 0],\n",
      "        ...,\n",
      "        [0, 0, 0, ..., 0, 0, 0],\n",
      "        [0, 0, 0, ..., 0, 0, 0],\n",
      "        [0, 0, 0, ..., 0, 0, 0]],\n",
      "\n",
      "       [[0, 0, 0, ..., 0, 0, 0],\n",
      "        [0, 0, 0, ..., 0, 0, 0],\n",
      "        [0, 0, 0, ..., 0, 0, 0],\n",
      "        ...,\n",
      "        [0, 0, 0, ..., 0, 0, 0],\n",
      "        [0, 0, 0, ..., 0, 0, 0],\n",
      "        [0, 0, 0, ..., 0, 0, 0]],\n",
      "\n",
      "       ...,\n",
      "\n",
      "       [[0, 0, 0, ..., 0, 0, 0],\n",
      "        [0, 0, 0, ..., 0, 0, 0],\n",
      "        [0, 0, 0, ..., 0, 0, 0],\n",
      "        ...,\n",
      "        [0, 0, 0, ..., 0, 0, 0],\n",
      "        [0, 0, 0, ..., 0, 0, 0],\n",
      "        [0, 0, 0, ..., 0, 0, 0]],\n",
      "\n",
      "       [[0, 0, 0, ..., 0, 0, 0],\n",
      "        [0, 0, 0, ..., 0, 0, 0],\n",
      "        [0, 0, 0, ..., 0, 0, 0],\n",
      "        ...,\n",
      "        [0, 0, 0, ..., 0, 0, 0],\n",
      "        [0, 0, 0, ..., 0, 0, 0],\n",
      "        [0, 0, 0, ..., 0, 0, 0]],\n",
      "\n",
      "       [[0, 0, 0, ..., 0, 0, 0],\n",
      "        [0, 0, 0, ..., 0, 0, 0],\n",
      "        [0, 0, 0, ..., 0, 0, 0],\n",
      "        ...,\n",
      "        [0, 0, 0, ..., 0, 0, 0],\n",
      "        [0, 0, 0, ..., 0, 0, 0],\n",
      "        [0, 0, 0, ..., 0, 0, 0]]], dtype=uint8), array([5, 0, 4, ..., 5, 6, 8], dtype=uint8)), (array([[[0, 0, 0, ..., 0, 0, 0],\n",
      "        [0, 0, 0, ..., 0, 0, 0],\n",
      "        [0, 0, 0, ..., 0, 0, 0],\n",
      "        ...,\n",
      "        [0, 0, 0, ..., 0, 0, 0],\n",
      "        [0, 0, 0, ..., 0, 0, 0],\n",
      "        [0, 0, 0, ..., 0, 0, 0]],\n",
      "\n",
      "       [[0, 0, 0, ..., 0, 0, 0],\n",
      "        [0, 0, 0, ..., 0, 0, 0],\n",
      "        [0, 0, 0, ..., 0, 0, 0],\n",
      "        ...,\n",
      "        [0, 0, 0, ..., 0, 0, 0],\n",
      "        [0, 0, 0, ..., 0, 0, 0],\n",
      "        [0, 0, 0, ..., 0, 0, 0]],\n",
      "\n",
      "       [[0, 0, 0, ..., 0, 0, 0],\n",
      "        [0, 0, 0, ..., 0, 0, 0],\n",
      "        [0, 0, 0, ..., 0, 0, 0],\n",
      "        ...,\n",
      "        [0, 0, 0, ..., 0, 0, 0],\n",
      "        [0, 0, 0, ..., 0, 0, 0],\n",
      "        [0, 0, 0, ..., 0, 0, 0]],\n",
      "\n",
      "       ...,\n",
      "\n",
      "       [[0, 0, 0, ..., 0, 0, 0],\n",
      "        [0, 0, 0, ..., 0, 0, 0],\n",
      "        [0, 0, 0, ..., 0, 0, 0],\n",
      "        ...,\n",
      "        [0, 0, 0, ..., 0, 0, 0],\n",
      "        [0, 0, 0, ..., 0, 0, 0],\n",
      "        [0, 0, 0, ..., 0, 0, 0]],\n",
      "\n",
      "       [[0, 0, 0, ..., 0, 0, 0],\n",
      "        [0, 0, 0, ..., 0, 0, 0],\n",
      "        [0, 0, 0, ..., 0, 0, 0],\n",
      "        ...,\n",
      "        [0, 0, 0, ..., 0, 0, 0],\n",
      "        [0, 0, 0, ..., 0, 0, 0],\n",
      "        [0, 0, 0, ..., 0, 0, 0]],\n",
      "\n",
      "       [[0, 0, 0, ..., 0, 0, 0],\n",
      "        [0, 0, 0, ..., 0, 0, 0],\n",
      "        [0, 0, 0, ..., 0, 0, 0],\n",
      "        ...,\n",
      "        [0, 0, 0, ..., 0, 0, 0],\n",
      "        [0, 0, 0, ..., 0, 0, 0],\n",
      "        [0, 0, 0, ..., 0, 0, 0]]], dtype=uint8), array([7, 2, 1, ..., 4, 5, 6], dtype=uint8)))\n"
     ]
    }
   ],
   "source": [
    "# from tensorflow.examples.tutorials.mnist import input_data\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "mnist = tf.keras.datasets.mnist\n",
    "print(tf.__version__)\n",
    "x = mnist.load_data()\n",
    "print(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(mnist.train.images.shape)\n",
    "# print(mnist.train.labels.shape)\n",
    "# print(mnist.validation.images.shape)\n",
    "# print(mnist.validation.labels.shape)\n",
    "# plt.figure(figsize=(10,10))\n",
    "# labels = []\n",
    "# with tf.Session() as sess:\n",
    "# #     l = tf.argmax(td.onmnist.train.labels[0:10],axis=0) \n",
    "#     l = tf.one_hot(mnist.train.labels[0:10],depth=10)\n",
    "#     labels = ['{}'.format(i) for i in sess.run(l)]\n",
    "# print(labels)\n",
    "# plt.subplot(3,3,1)\n",
    "# for idx in range(1,10):\n",
    "#     plt.subplot(3,3,idx)\n",
    "#     plt.title('[{}]'.format(mnist.train.labels[idx]),fontsize=16)\n",
    "#     plt.imshow(mnist.train.images[idx].reshape((28,28)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "\n",
    "def create_dense(input_num,output_num,neuron = [],graphic=None,weight_name=None,weight_lamda=0.01,init = tf.zeros_initializer()):\n",
    "    x = tf.placeholder(shape=[None,input_num],dtype=tf.float32)\n",
    "    if graphic is None and weight_lamda :\n",
    "        graphic = tf.get_default_graph()\n",
    "    \n",
    "    if neuron:\n",
    "        p_num = [input_num]\n",
    "        p_num +=neuron\n",
    "        n_num  = neuron.copy()\n",
    "        n_num.append(output_num)\n",
    "        hide = None\n",
    "        layer_deep = len(p_num)\n",
    "        for i,nums in enumerate(zip(p_num,n_num)):\n",
    "            print('[layer={} shape={}]'.format(i,nums))\n",
    "            weight =  tf.Variable(init(shape=nums))\n",
    "            tf.add_to_collection(weight_name,tf.contrib.layers.l2_regularizer(weight_lamda)(weight))\n",
    "            bias = tf.Variable(init(shape=[nums[1]]))                     \n",
    "            if hide is None:\n",
    "                hide = tf.matmul(x,weight) + bias\n",
    "            else :\n",
    "                hide = tf.matmul(hide,weight) +bias\n",
    "                if i != (layer_deep-1):\n",
    "                    hide = tf.nn.relu(hide)\n",
    "                \n",
    "        return hide,x\n",
    "        \n",
    "    else:\n",
    "        weight =tf.Variable(tf.ones_initializer()(shape=[input_num,output_num]))\n",
    "        bias = tf.Variable(tf.zeros_initializer()(shape=[output_num]))\n",
    "        return  tf.matmul(x,weight)  + bias , x    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'float'>\n",
      "0.1\n"
     ]
    }
   ],
   "source": [
    "x = 1e-5\n",
    "print(type(x))\n",
    "print(x* 10000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[layer=0 shape=(784, 128)]\n",
      "[layer=1 shape=(128, 10)]\n",
      "[i=0][entropy_loss=2217.3056640625][train_accury0.1875][test_accuracy=0.11630000174045563]\n",
      "[i=100][entropy_loss=1515.994384765625][train_accury0.78125][test_accuracy=0.8016999959945679]\n",
      "[i=200][entropy_loss=1044.21240234375][train_accury0.859375][test_accuracy=0.8748999834060669]\n",
      "[i=300][entropy_loss=725.0321044921875][train_accury0.90625][test_accuracy=0.8920000195503235]\n",
      "[i=400][entropy_loss=508.310302734375][train_accury0.9375][test_accuracy=0.8982999920845032]\n",
      "[i=500][entropy_loss=361.1343994140625][train_accury0.828125][test_accuracy=0.9024999737739563]\n",
      "[i=600][entropy_loss=260.4704895019531][train_accury0.9375][test_accuracy=0.9057999849319458]\n",
      "[i=700][entropy_loss=191.41110229492188][train_accury0.984375][test_accuracy=0.9082000255584717]\n",
      "[i=800][entropy_loss=143.8174285888672][train_accury0.96875][test_accuracy=0.9067999720573425]\n",
      "[i=900][entropy_loss=110.9605941772461][train_accury0.859375][test_accuracy=0.9093000292778015]\n",
      "[i=1000][entropy_loss=87.88864135742188][train_accury0.921875][test_accuracy=0.9157000184059143]\n",
      "[i=1100][entropy_loss=71.76554870605469][train_accury0.921875][test_accuracy=0.9117000102996826]\n",
      "[i=1200][entropy_loss=60.311283111572266][train_accury0.84375][test_accuracy=0.9120000004768372]\n",
      "[i=1300][entropy_loss=51.737064361572266][train_accury0.9375][test_accuracy=0.9115999937057495]\n",
      "[i=1400][entropy_loss=45.551448822021484][train_accury0.9375][test_accuracy=0.9158999919891357]\n",
      "[i=1500][entropy_loss=41.172245025634766][train_accury0.875][test_accuracy=0.9093000292778015]\n",
      "[i=1600][entropy_loss=37.688514709472656][train_accury0.875][test_accuracy=0.9143999814987183]\n",
      "[i=1700][entropy_loss=35.086490631103516][train_accury0.90625][test_accuracy=0.9182999730110168]\n",
      "[i=1800][entropy_loss=32.8811149597168][train_accury0.875][test_accuracy=0.916700005531311]\n",
      "[i=1900][entropy_loss=31.18805694580078][train_accury0.90625][test_accuracy=0.9154999852180481]\n",
      "[i=2000][entropy_loss=29.687435150146484][train_accury0.953125][test_accuracy=0.9192000031471252]\n"
     ]
    }
   ],
   "source": [
    "learning_rate = 0.1\n",
    "trainig_step = 1000\n",
    "batch_size = 32\n",
    "x_dim = mnist.train.images.shape[1]\n",
    "y_dim = 10\n",
    "y = tf.placeholder(shape=[None,y_dim],dtype=tf.int32)\n",
    "init = tf.random_normal_initializer(seed=0,stddev=0.01)\n",
    "# init = tf.random_uniform_initializer(0,1)\n",
    "# init = tf.ones_initializer()\n",
    "# init =tf.zeros_initializer()\n",
    "\n",
    "\n",
    "logits ,x = create_dense(x_dim,y_dim,[128],init=init,weight_name='weight_lamda',weight_lamda= e^-7)\n",
    "\n",
    "#Optimizer\n",
    "loss =  tf.nn.softmax_cross_entropy_with_logits_v2(logits=logits, labels=y,name='Loss') \n",
    "cross_entropy_loss_l2reg = tf.reduce_mean(loss) + tf.add_n(tf.get_collection('weight_lamda')) \n",
    "optimizer = tf.train.GradientDescentOptimizer(learning_rate=learning_rate).minimize(cross_entropy_loss_l2reg)\n",
    "\n",
    "#test \n",
    "pred = tf.nn.softmax(logits)\n",
    "\n",
    "#恢复one_hot to int\n",
    "sun = tf.reduce_sum(y * tf.range(0,10,1,dtype = tf.int32),axis=-1)\n",
    "\n",
    "num = tf.cast(sun,tf.int64)\n",
    "correct_pred = tf.equal(tf.argmax(pred, 1), tf.cast(num,tf.int64))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    for i  in range(trainig_step+1):\n",
    "        tx , ty = mnist.train.next_batch(batch_size)\n",
    "        a,_,lose = sess.run([accuracy,optimizer,cross_entropy_loss_l2reg ],feed_dict={x:tx,y:ty})\n",
    "        \n",
    "        if i %100 == 0:\n",
    "            validate_accuracy = sess.run(accuracy,feed_dict={x:mnist.test.images,y:mnist.test.labels})\n",
    "            print('[i={}][entropy_loss={}][train_accury{}][test_accuracy={}]'.format(i,lose,a,validate_accuracy))\n",
    "\n",
    "               "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'e' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-1-8a308b54420d>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0me\u001b[0m\u001b[0;34m^\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m7\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'e' is not defined"
     ]
    }
   ],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
